#!/usr/bin/perl -w
use strict;
use Getopt::Long;
use FindBin;
use lib ($FindBin::Bin, "$FindBin::Bin/../perl_modules");
use die_error;

#88888888888888888888888888888888888888888888888888888888888888888888888888888888

#	Usage

#88888888888888888888888888888888888888888888888888888888888888888888888888888888

my $usage = <<'USAGE';

USAGE:

    break_into_chunks.pl
				--every 				CHUNK_SIZE 		(e.g. 2000)
				--min_chunk_size		MIN_CHUNK_SIZE  (e.g. 200)
				--min_resulting_files	FILE_COUNT      (e.g. 50)
				--base_name 			file.ext
				--input_file 			INPUT_FILE

	Breaks input into chunks and write to file.NN.ext where NN = an increasing
	integer. e.g. file.1.ext file.2.ext file.3.ext etc.

	This programme also handles situations when the number of lines is smaller than
		expected. It will break into small chunks if there are few lines, unless this
		results in tiny chunks:

	1) When line_count >= CHUNK_SIZE * FILE_COUNT then
		chunk size = CHUNK_SIZE.

	2) When line_count < CHUNK_SIZE * FILE_COUNT
		chunk size = line_count / FILE_COUNT

	3) When line_count < MIN_CHUNK_SIZE * FILE_COUNT
		chunk_size = MIN_CHUNK_SIZE
USAGE


#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888

#	Get options

#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
# mandatory

# optional parameters
my $files = '';
my $input_file;
my $target_chunk_size;
my $min_chunk_size;
my $min_resulting_files;
my $base_name;
my $help;
GetOptions(
			'every=i'					=> \$target_chunk_size,
			'min_resulting_files=i'		=> \$min_resulting_files,
			'min_chunk_size=i'			=> \$min_chunk_size,
			'base_name=s'				=> \$base_name,
			'input_file|i=s'			=> \$input_file,
			'help'						=> \$help);


die $usage if ($help);
die_error "Please enter target chunk size (number of lines) via --every.\n".$usage
		unless ($target_chunk_size);
die_error "Please enter minimum chunk size (number of lines) via --min_chunk_size.\n".$usage
		unless ($target_chunk_size);
die_error "Please enter target minimum number of output files via --min_resulting_files.\n".$usage
		unless ($target_chunk_size);
die_error "Please enter the source file via --input_file.\n".$usage
		unless ($input_file);

die_error "Please supply output files path via --base_name.\n".$usage
		unless ($base_name);

#
# count lines
# 
print "\tGetting size of original file...\n";
my $line_count = 0;
{
	open_or_die(*FILE, "$input_file");
	$line_count += tr/\n/\n/ while sysread(FILE, $_, 2 ** 16);
	print "\t$line_count lines in $input_file\n";
}

#
# calculate chunk_size
#
my $chunk_size;

# really starting tiny file
if ($line_count < $min_resulting_files * $min_chunk_size)
{
	$chunk_size = $min_chunk_size;
}

# quite small file
elsif ($line_count < $target_chunk_size * $min_resulting_files)
{
	$chunk_size = int(($line_count / $min_resulting_files) + 1);
}
else
{
	$chunk_size = $target_chunk_size;
}
print "\tBreaking up into files of $chunk_size lines:\n";


my ($path, $ext);
if ($base_name =~ /(.+)(\..+)/)
{
	($path, $ext) = ($1, $2);
}
else
{
	($path, $ext) = ($base_name, "");
}

my $cnt_pair = 0;
print "\t[";
my $cnt_pair_file = 0;
open_or_die(*FILE, "$input_file");
while (<FILE>)
{
	if ($cnt_pair % $chunk_size == 0)
	{
		close PAIR;
		open PAIR, ">$path.$cnt_pair_file$ext" or
			die "Error\n\tCould not open $path.$cnt_pair$ext\n$!";
		$cnt_pair_file++;
		if ($cnt_pair_file % 500 == 0)
		{
			print "\n\t" if ($cnt_pair_file / 500) % 50 == 0;
			print ".";
		}

	}
	$cnt_pair++;
	print PAIR $_;
}
print "]\n";
print "\t$cnt_pair_file files produced.\n";


